import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class Thread_FindRepeatFile {
	private static Logger log = LoggerFactory.getLogger(Thread_FindRepeatFile.class);

	private static Object synObj = new Object();
	private static final int PAGE_SIZE = 1000;
	private static int CNT = 0;
	private static final int CNT_PRINT = 200;
	private static final int MAX_THREAD_NUM = 120;

	public static void main(String srgs[]) throws Exception {
		long time = System.currentTimeMillis();
		Map<String, List<String>> map = findRepeatFile(new File("G:\\work\\DSDF\\建行文档-DSDF"));
		printRepeatFile(map);
		log.error("time over !" + (System.currentTimeMillis() - time));
	}

	public static void printRepeatFile(Map<String, List<String>> map) {
		Iterator<Map.Entry<String, List<String>>> it = map.entrySet().iterator();
		Map.Entry<String, List<String>> entity = null;
		List<String> list = null;
		log.error("find repeat file count for : " + map.size());
		int cnt = 0;
		while (it.hasNext()) {
			entity = it.next();
			log.error("==========" + entity.getKey() + "========== " + (++cnt));
			list = entity.getValue();
			for (int i = 0; list != null && i < list.size(); i++) {
				log.error(list.get(i));
			}
		}
	}

	public static Map<String, List<String>> findRepeatFile(File root) throws Exception {
		List<String> files = new ArrayList<String>();
		find(root, files);
		Map<String, List<String>> map = new Hashtable<String, List<String>>();
		int size = files.size();
		if (size == 0)
			return map;
		int pageSize = PAGE_SIZE;
		int cnt = size <= PAGE_SIZE ? 1 : (size / PAGE_SIZE + (size % PAGE_SIZE == 0 ? 0 : 1));
		if (cnt > MAX_THREAD_NUM) {
			cnt = MAX_THREAD_NUM;
			pageSize = size / MAX_THREAD_NUM + (size % MAX_THREAD_NUM == 0 ? 0 : 1);
		}
		int end = 0;
		int start = 0;
		List<String> items = null;
		for (int i = 0; i < cnt; i++) {
			start = i * pageSize;
			end = start + ((i + 1) * pageSize > size ? size % pageSize : pageSize);
			items = files.subList(start, end);
			md5(items, map, i);
		}
		while (cnt != cntNow(false)) {
			Thread.sleep(1000 * 3);
		}
		items = null;
		files = null;
		Map<String, List<String>> rst = new HashMap<String, List<String>>();
		Set<Map.Entry<String, List<String>>> set = map.entrySet();
		Iterator<Map.Entry<String, List<String>>> it = set.iterator();
		Map.Entry<String, List<String>> entity = null;
		while (it.hasNext()) {
			entity = it.next();
			if (entity.getValue().size() > 1)
				rst.put(entity.getKey(), entity.getValue());
		}
		return rst;

	}

	private static void md5(final List<String> files, final Map<String, List<String>> map, final int cnt)
			throws Exception {
		Thread t = new Thread(new Runnable() {
			@Override
			public void run() {
				Thread.currentThread().setName("Md5thread-" + cnt);
				String md5 = null;
				for (int i = 0; files != null && i < files.size(); i++) {
					md5 = md5(new File(files.get(i)));
					synchronized (synObj) {
						List<String> list = map.get(md5);
						if (list == null) {
							list = new ArrayList<String>();
							map.put(md5, list);
						}
						list.add(files.get(i));
						if (i >= CNT_PRINT && i % CNT_PRINT == 0 || i == files.size() - 1)
							log.error(Thread.currentThread().getName() + " loop for:" + i);
					}
				}
				cntNow(true);
			}
		});
		t.start();
	}

	private static void find(File file, List<String> files) {
		if (file.isDirectory()) {
			File[] items = file.listFiles();
			for (int i = 0; items != null && i < items.length; i++) {
				find(items[i], files);
			}
		} else {
			files.add(file.getAbsolutePath());
		}

	}

	private static String md5(File file) {
		BigInteger bi = null;
		FileInputStream fis = null;
		try {
			byte[] buffer = new byte[1024 * 1024 * 3];
			int len = 0;
			MessageDigest md = MessageDigest.getInstance("MD5");
			fis = new FileInputStream(file);
			while ((len = fis.read(buffer)) != -1) {
				md.update(buffer, 0, len);
			}
			fis.close();
			byte[] b = md.digest();
			bi = new BigInteger(1, b);
			buffer = null;
		} catch (Exception e) {
			throw new RuntimeException("read file[" + file.getAbsolutePath() + " ] fail.", e);
		} finally {
			try {
				fis.close();
			} catch (IOException ex) {
				log.error("close fail", ex);
			}
		}
		return bi.toString(16);
	}

	private static int cntNow(boolean add) {
		synchronized (synObj) {
			if (add)
				return ++CNT;
			return CNT;
		}
	}
}
